{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-1-d4bc1db9c8ea>:66: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From d:\\python3\\tfgpu\\dl+\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From d:\\python3\\tfgpu\\dl+\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ../../../datasets/MNIST_data\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From d:\\python3\\tfgpu\\dl+\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ../../../datasets/MNIST_data\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From d:\\python3\\tfgpu\\dl+\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting ../../../datasets/MNIST_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting ../../../datasets/MNIST_data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From d:\\python3\\tfgpu\\dl+\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "After 1 training step(s), loss on training batch is 3.34079.\n",
      "After 1001 training step(s), loss on training batch is 0.204589.\n",
      "After 2001 training step(s), loss on training batch is 0.17983.\n",
      "After 3001 training step(s), loss on training batch is 0.131656.\n",
      "After 4001 training step(s), loss on training batch is 0.114602.\n",
      "After 5001 training step(s), loss on training batch is 0.109876.\n",
      "After 6001 training step(s), loss on training batch is 0.104787.\n",
      "After 7001 training step(s), loss on training batch is 0.0878698.\n",
      "After 8001 training step(s), loss on training batch is 0.0772841.\n",
      "After 9001 training step(s), loss on training batch is 0.0761682.\n",
      "After 10001 training step(s), loss on training batch is 0.0685607.\n",
      "After 11001 training step(s), loss on training batch is 0.0700717.\n",
      "After 12001 training step(s), loss on training batch is 0.0638655.\n",
      "After 13001 training step(s), loss on training batch is 0.0604035.\n",
      "After 14001 training step(s), loss on training batch is 0.0541286.\n",
      "After 15001 training step(s), loss on training batch is 0.0471391.\n",
      "After 16001 training step(s), loss on training batch is 0.0471977.\n",
      "After 17001 training step(s), loss on training batch is 0.0545788.\n",
      "After 18001 training step(s), loss on training batch is 0.0482854.\n",
      "After 19001 training step(s), loss on training batch is 0.0397977.\n",
      "After 20001 training step(s), loss on training batch is 0.0392055.\n",
      "After 21001 training step(s), loss on training batch is 0.0394956.\n",
      "After 22001 training step(s), loss on training batch is 0.0438369.\n",
      "After 23001 training step(s), loss on training batch is 0.0368074.\n",
      "After 24001 training step(s), loss on training batch is 0.0365462.\n",
      "After 25001 training step(s), loss on training batch is 0.037705.\n",
      "After 26001 training step(s), loss on training batch is 0.0351096.\n",
      "After 27001 training step(s), loss on training batch is 0.04149.\n",
      "After 28001 training step(s), loss on training batch is 0.0347917.\n",
      "After 29001 training step(s), loss on training batch is 0.0333904.\n"
     ]
    },
    {
     "ename": "SystemExit",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[1;31mSystemExit\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python3\\tfgpu\\dl+\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2918: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "# %load mnist_train.py\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import os\n",
    "\n",
    "# 加载mnist_inference.py中定义的常量和前向传播的函数\n",
    "import mnist_inference\n",
    "\n",
    "# 配置神经网络的参数\n",
    "BATCH_SIZE = 100\n",
    "LEARNING_RATE_BASE = 0.8\n",
    "LEARNING_RATE_DECAY = 0.99\n",
    "REGULARIZATION_RATE = 0.0001\n",
    "TRAINING_STEPS = 30000\n",
    "MOVING_AVERAGE_DECAY = 0.99\n",
    "\n",
    "# 模型保存的路径和文件名\n",
    "MODEL_SAVE_PATH=\"MNIST_model/\"\n",
    "MODEL_NAME=\"mnist_model\"\n",
    "\n",
    "\n",
    "def train(mnist):\n",
    "    # 1. 定义神经网络的输入输出（网络参数由于mnist_inference函数，定义在了前向传播中）\n",
    "    x = tf.placeholder(tf.float32, [None, mnist_inference.INPUT_NODE], name='x-input')\n",
    "    y_ = tf.placeholder(tf.float32, [None, mnist_inference.OUTPUT_NODE], name='y-input')\n",
    "\n",
    "    # 2. 定义前向传播、损失函数、反向传播\n",
    "    # 前向传播（包含定义神经网络参数）\n",
    "    regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)\n",
    "    y = mnist_inference.inference(x, regularizer)\n",
    "    # 损失函数\n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    "    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)\n",
    "    variables_averages_op = variable_averages.apply(tf.trainable_variables())\n",
    "    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))\n",
    "    cross_entropy_mean = tf.reduce_mean(cross_entropy)\n",
    "    loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))\n",
    "    # 反向传播（即优化算法）\n",
    "    learning_rate = tf.train.exponential_decay(\n",
    "        LEARNING_RATE_BASE,\n",
    "        global_step,\n",
    "        mnist.train.num_examples / BATCH_SIZE, LEARNING_RATE_DECAY,\n",
    "        staircase=True)\n",
    "    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)\n",
    "    with tf.control_dependencies([train_step, variables_averages_op]):\n",
    "        train_op = tf.no_op(name='train')\n",
    "    \n",
    "    # 3. 建立会话，训练\n",
    "    saver = tf.train.Saver()   # 先初始化持久类\n",
    "    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)   # 给当前session分配固定数量显存\n",
    "    with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:\n",
    "        tf.global_variables_initializer().run()\n",
    "        # 训练过程中，不再测试模型在验证数据上的表现，验证和测试的过程会有一个独立的程序来完成\n",
    "        for i in range(TRAINING_STEPS):\n",
    "            xs, ys = mnist.train.next_batch(BATCH_SIZE)\n",
    "            _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: xs, y_: ys})\n",
    "            # 每1000轮保存一次模型\n",
    "            if i % 1000 == 0:\n",
    "                # 这里仅输出模型在当前训练batch上的损失函数，大概来了解训练情况\n",
    "                print(\"After %d training step(s), loss on training batch is %g.\" % (step, loss_value))\n",
    "                # 保存当前模型，注意这里给出了global_step参数，这样可以让每个被保存模型的文件名末尾加上训练轮数，如：model.ckpt-1000\n",
    "                saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)\n",
    "\n",
    "\n",
    "def main(argv=None):\n",
    "    mnist = input_data.read_data_sets(\"../../../datasets/MNIST_data\", one_hot=True)\n",
    "    train(mnist)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    tf.app.run()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`An exception has occurred, use %tb to see the full traceback.\n",
    "SystemExit`\n",
    "\n",
    "是因为在IPython下才会发生，使用命令行界面并不会。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
